{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notebook written by [Zhedong Zheng](https://github.com/zhedongzheng)\n",
    "\n",
    "<img src=\"img/self_attn.png\" width=\"200\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    'batch_size': 64,\n",
    "    'text_iter_step': 25,\n",
    "    'seq_len': 200,\n",
    "    'hidden_dim': 128,\n",
    "    'num_head': 8,\n",
    "    'n_hidden_layer': 2,\n",
    "    'display_step': 10,\n",
    "    'generate_step': 100,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_text(file_path):\n",
    "    with open(file_path) as f:\n",
    "        text = f.read()\n",
    "    \n",
    "    char2idx = {c: i+3 for i, c in enumerate(set(text))}\n",
    "    char2idx['<pad>'] = 0\n",
    "    char2idx['<start>'] = 1\n",
    "    char2idx['<end>'] = 2\n",
    "    \n",
    "    ints = np.array([char2idx[char] for char in list(text)])\n",
    "    return ints, char2idx\n",
    "\n",
    "def next_batch(ints):\n",
    "    len_win = params['seq_len'] * params['batch_size']\n",
    "    for i in range(0, len(ints)-len_win, params['text_iter_step']):\n",
    "        clip = ints[i: i+len_win]\n",
    "        yield clip.reshape([params['batch_size'], params['seq_len']])\n",
    "        \n",
    "def input_fn(ints):\n",
    "    dataset = tf.data.Dataset.from_generator(\n",
    "        lambda: next_batch(ints), tf.int32, tf.TensorShape([None, params['seq_len']]))\n",
    "    iterator = dataset.make_one_shot_iterator()\n",
    "    return iterator.get_next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def start_sent(x):\n",
    "    _x = tf.fill([tf.shape(x)[0], 1], params['char2idx']['<start>'])\n",
    "    return tf.concat([_x, x], 1)\n",
    "\n",
    "def end_sent(x):\n",
    "    _x = tf.fill([tf.shape(x)[0], 1], params['char2idx']['<end>'])\n",
    "    return tf.concat([x, _x], 1)\n",
    "\n",
    "def embed_seq(x, vocab_sz, embed_dim, name, zero_pad=False, scale=False):\n",
    "    embedding = tf.get_variable(name, [vocab_sz, embed_dim])\n",
    "    if zero_pad:\n",
    "        embedding = tf.concat([tf.zeros([1, embed_dim]), embedding[1:, :]], 0)\n",
    "    x = tf.nn.embedding_lookup(embedding, x)\n",
    "    if scale:\n",
    "        x = x * np.sqrt(embed_dim)\n",
    "    return x\n",
    "\n",
    "def position_embedding(inputs):\n",
    "    T = inputs.get_shape().as_list()[1]\n",
    "    x = tf.range(T)                            # (T)\n",
    "    x = tf.expand_dims(x, 0)                   # (1, T)\n",
    "    x = tf.tile(x, [tf.shape(inputs)[0], 1])   # (N, T)\n",
    "    return embed_seq(x, T, params['hidden_dim'], 'position_embedding')\n",
    "\n",
    "def layer_norm(inputs, epsilon=1e-8):\n",
    "    mean, variance = tf.nn.moments(inputs, [-1], keep_dims=True)\n",
    "    normalized = (inputs - mean) / (tf.sqrt(variance + epsilon))\n",
    "    \n",
    "    params_shape = inputs.get_shape()[-1:]\n",
    "    gamma = tf.get_variable('gamma', params_shape, tf.float32, tf.ones_initializer())\n",
    "    beta = tf.get_variable('beta', params_shape, tf.float32, tf.zeros_initializer())\n",
    "    \n",
    "    return gamma * normalized + beta\n",
    "\n",
    "def self_attention(inputs, is_training, activation=None):\n",
    "    num_units = params['hidden_dim']\n",
    "    num_heads = params['num_head']\n",
    "    T_q = T_k = inputs.get_shape().as_list()[1]\n",
    "\n",
    "    Q_K_V = tf.layers.dense(inputs, 3*num_units, activation)\n",
    "    Q, K, V = tf.split(Q_K_V, 3, -1)\n",
    "    Q_ = tf.concat(tf.split(Q, num_heads, axis=2), 0)                         \n",
    "    K_ = tf.concat(tf.split(K, num_heads, axis=2), 0)                        \n",
    "    V_ = tf.concat(tf.split(V, num_heads, axis=2), 0)                         \n",
    "\n",
    "    align = tf.matmul(Q_, tf.transpose(K_, [0,2,1]))                               \n",
    "    align = align / np.sqrt(K_.get_shape().as_list()[-1])\n",
    "\n",
    "    paddings = tf.fill(tf.shape(align), float('-inf'))         \n",
    "    lower_tri = tf.ones([T_q, T_k])                                                \n",
    "    lower_tri = tf.linalg.LinearOperatorLowerTriangular(lower_tri).to_dense()      \n",
    "    masks = tf.tile(tf.expand_dims(lower_tri,0), [tf.shape(align)[0],1,1])       \n",
    "    align = tf.where(tf.equal(masks, 0), paddings, align)               \n",
    "\n",
    "    align = tf.nn.softmax(align)                                                  \n",
    "    align = tf.layers.dropout(align, 0.1, training=is_training)           \n",
    "    x = tf.matmul(align, V_)                                                 \n",
    "    x = tf.concat(tf.split(x, num_heads, axis=0), 2)              \n",
    "    x += inputs                                                                \n",
    "    x = layer_norm(x)                                                 \n",
    "    return x\n",
    "\n",
    "def ffn(inputs, activation=tf.nn.relu):\n",
    "    x = tf.layers.conv1d(inputs, 4*params['hidden_dim'], 1, activation=activation)\n",
    "    x = tf.layers.conv1d(x, params['hidden_dim'], 1, activation=None)\n",
    "    x += inputs\n",
    "    x = layer_norm(x)\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward(inputs, reuse, is_training):\n",
    "    inputs = start_sent(inputs)\n",
    "    with tf.variable_scope('model', reuse=reuse):\n",
    "        x = embed_seq(inputs, params['vocab_size'], params['hidden_dim'], 'word_embedding',\n",
    "                      zero_pad=True, scale=True)\n",
    "        x += position_embedding(x)\n",
    "        x = tf.layers.dropout(x, 0.1, training=is_training)\n",
    "        \n",
    "        for i in range(params['n_hidden_layer']):\n",
    "            with tf.variable_scope('attn_%d'%i, reuse=reuse):\n",
    "                x = self_attention(x, is_training)\n",
    "            with tf.variable_scope('ffn_%d'%i, reuse=reuse):\n",
    "                x = ffn(x)\n",
    "        \n",
    "        logits = tf.layers.dense(x, params['vocab_size'])\n",
    "    return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def autoregressive():\n",
    "    def cond(i, x, temp):\n",
    "        return i < params['seq_len']\n",
    "\n",
    "    def body(i, x, temp):\n",
    "        logits = forward(x, reuse=True, is_training=False)\n",
    "        ids = tf.argmax(logits, -1, output_type=tf.int32)[:, i]\n",
    "        ids = tf.expand_dims(ids, -1)\n",
    "\n",
    "        temp = tf.concat([temp[:, 1:], ids], -1)\n",
    "\n",
    "        x = tf.concat([temp[:, -(i+1):], temp[:, :-(i+1)]], -1)\n",
    "        x = tf.reshape(x, [1, params['seq_len']])\n",
    "        i += 1\n",
    "        return i, x, temp\n",
    "\n",
    "    x = tf.zeros([1, params['seq_len']], tf.int32)\n",
    "    _, res, _ = tf.while_loop(cond, body, [tf.constant(0), x, x])\n",
    "    \n",
    "    return res[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocabulary size: 86\n"
     ]
    }
   ],
   "source": [
    "ints, params['char2idx'] = parse_text('../temp/anna.txt')\n",
    "params['vocab_size'] = len(params['char2idx'])\n",
    "params['idx2char'] = {i: c for c, i in params['char2idx'].items()}\n",
    "print('Vocabulary size:', params['vocab_size'])\n",
    "\n",
    "X = input_fn(ints)\n",
    "logits = forward(X, reuse=False, is_training=True)\n",
    "\n",
    "ops = {}\n",
    "ops['global_step'] = tf.Variable(0, trainable=False)\n",
    "\n",
    "targets = end_sent(X)\n",
    "ops['loss'] = tf.reduce_mean(tf.contrib.seq2seq.sequence_loss(\n",
    "    logits = logits,\n",
    "    targets = targets,\n",
    "    weights = tf.to_float(tf.ones_like(targets))))\n",
    "\n",
    "ops['train'] = tf.train.AdamOptimizer().minimize(ops['loss'], global_step=ops['global_step'])\n",
    "\n",
    "ops['generate'] = autoregressive()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1 | Loss 4.827\n",
      "Step 10 | Loss 2.809\n",
      "Step 20 | Loss 2.623\n",
      "Step 30 | Loss 2.537\n",
      "Step 40 | Loss 2.492\n",
      "Step 50 | Loss 2.464\n",
      "Step 60 | Loss 2.442\n",
      "Step 70 | Loss 2.420\n",
      "Step 80 | Loss 2.408\n",
      "Step 90 | Loss 2.394\n",
      "Step 100 | Loss 2.379\n",
      "\n",
      " he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he he h\n",
      "\n",
      "Step 110 | Loss 2.361\n",
      "Step 120 | Loss 2.350\n",
      "Step 130 | Loss 2.331\n",
      "Step 140 | Loss 2.306\n",
      "Step 150 | Loss 2.287\n",
      "Step 160 | Loss 2.268\n",
      "Step 170 | Loss 2.245\n",
      "Step 180 | Loss 2.231\n",
      "Step 190 | Loss 2.201\n",
      "Step 200 | Loss 2.177\n",
      "\n",
      " he sin the he s se cof his his his his his his his his his his his s his his his hise his his se se hise his s hise se se con hise hise hise se se con hise hise hise hise hise his his se hise s hise \n",
      "\n",
      "Step 210 | Loss 2.174\n",
      "Step 220 | Loss 2.163\n",
      "Step 230 | Loss 2.123\n",
      "Step 240 | Loss 2.107\n",
      "Step 250 | Loss 2.112\n",
      "Step 260 | Loss 2.067\n",
      "Step 270 | Loss 2.038\n",
      "Step 280 | Loss 2.038\n",
      "Step 290 | Loss 2.019\n",
      "Step 300 | Loss 2.003\n",
      "\n",
      " and the sin tour the stooo f he list he list ist ist in on on thist his oure four f hist le histhist\n",
      "t hise le histhis l------t mon st sthe sthe st sthe sthe sthe sthe st st sthe sthe st stoooooooooo\n",
      "\n",
      "Step 310 | Loss 1.970\n",
      "Step 320 | Loss 1.955\n",
      "Step 330 | Loss 1.915\n",
      "Step 340 | Loss 1.902\n",
      "Step 350 | Loss 1.896\n",
      "Step 360 | Loss 1.869\n",
      "Step 370 | Loss 1.850\n",
      "Step 380 | Loss 1.832\n",
      "Step 390 | Loss 1.810\n",
      "Step 400 | Loss 1.790\n",
      "\n",
      " he said to ther se for his his his his his his his his his his his his his his his our his oust hist his his our our histhist his our onconse our histhist\n",
      "conse onse ofathiate his his ond hise his hi\n",
      "\n",
      "Step 410 | Loss 1.782\n",
      "Step 420 | Loss 1.751\n",
      "Step 430 | Loss 1.729\n",
      "Step 440 | Loss 1.731\n",
      "Step 450 | Loss 1.696\n",
      "Step 460 | Loss 1.683\n",
      "Step 470 | Loss 1.667\n",
      "Step 480 | Loss 1.658\n",
      "Step 490 | Loss 1.632\n",
      "Step 500 | Loss 1.616\n",
      "\n",
      " and the sat sat sating stin spo tureal to to the se hat se sto rve he he her helushovews tho thererer helushon t he the could t the the\n",
      "wo the there the he he haprerestond wing hereng here he\n",
      "m, and \n",
      "\n",
      "Step 510 | Loss 1.600\n",
      "Step 520 | Loss 1.573\n",
      "Step 530 | Loss 1.540\n",
      "Step 540 | Loss 1.543\n",
      "Step 550 | Loss 1.519\n",
      "Step 560 | Loss 1.495\n",
      "Step 570 | Loss 1.476\n",
      "Step 580 | Loss 1.450\n",
      "Step 590 | Loss 1.440\n",
      "Step 600 | Loss 1.408\n",
      "\n",
      " shis was impossible to an the ouse t to rathe at at at ratin atin.\n",
      "\n",
      "\"But monst the st she she stre sat sathemo stimeatimo re reattivele gre on olen her ove him, an tan and totat grand\n",
      "t ther salinger\n",
      "\n",
      "Step 610 | Loss 1.402\n",
      "Step 620 | Loss 1.409\n",
      "Step 630 | Loss 1.373\n",
      "Step 640 | Loss 1.335\n",
      "Step 650 | Loss 1.310\n",
      "Step 660 | Loss 1.296\n",
      "Step 670 | Loss 1.306\n",
      "Step 680 | Loss 1.270\n",
      "Step 690 | Loss 1.253\n",
      "Step 700 | Loss 1.224\n",
      "\n",
      " her the sawas ing he was she was douthe ayss ll t the mer me ther dooorsooouge t hin the date alo at ay\n",
      "ther.\n",
      "\n",
      "\"I tolllll me me came came tonome, hint can ce therek oussof\n",
      "t the there be fatherelere \n",
      "\n",
      "Step 710 | Loss 1.209\n",
      "Step 720 | Loss 1.194\n",
      "Step 730 | Loss 1.214\n",
      "Step 740 | Loss 1.190\n",
      "Step 750 | Loss 1.167\n",
      "Step 760 | Loss 1.163\n",
      "Step 770 | Loss 1.142\n",
      "Step 780 | Loss 1.139\n",
      "Step 790 | Loss 1.124\n",
      "Step 800 | Loss 1.113\n",
      "\n",
      " she was could tome the whe alas ind to thin the walker\n",
      "themer herooough blder bethe ck couldry iales hall wimes thrin thin this was ther thempin din to herssssssssioussiousiot e gher her ad and and a\n",
      "\n",
      "Step 810 | Loss 1.104\n",
      "Step 820 | Loss 1.108\n",
      "Step 830 | Loss 1.070\n",
      "Step 840 | Loss 1.073\n",
      "Step 850 | Loss 1.066\n",
      "Step 860 | Loss 1.073\n",
      "Step 870 | Loss 1.045\n",
      "Step 880 | Loss 1.022\n",
      "Step 890 | Loss 1.034\n",
      "Step 900 | Loss 1.017\n",
      "\n",
      " she was che win the wall fres he ad gond not a of he goiner he her wn as.\n",
      "\n",
      "\n",
      "\"Darya Allexandrovn med ly! be Herrrrendably how hathis\n",
      "wared foried thenthow he he breard, and fow his heaching an his\n",
      "spr\n",
      "\n",
      "Step 910 | Loss 1.003\n",
      "Step 920 | Loss 1.007\n",
      "Step 930 | Loss 0.984\n",
      "Step 940 | Loss 0.991\n",
      "Step 950 | Loss 1.000\n",
      "Step 960 | Loss 0.981\n",
      "Step 970 | Loss 0.982\n",
      "Step 980 | Loss 0.966\n",
      "Step 990 | Loss 0.947\n",
      "Step 1000 | Loss 0.943\n",
      "\n",
      " she had sallas red and agonced insting tin to the towo the theringshe the thr down the the nour driby theng thererding on of he the\n",
      "oonars. In w him sen rad then the be theroard the din, nowe it the \n",
      "\n",
      "Step 1010 | Loss 0.942\n",
      "Step 1020 | Loss 0.915\n",
      "Step 1030 | Loss 0.924\n",
      "Step 1040 | Loss 0.903\n",
      "Step 1050 | Loss 0.912\n",
      "Step 1060 | Loss 0.893\n",
      "Step 1070 | Loss 0.906\n",
      "Step 1080 | Loss 0.905\n",
      "Step 1090 | Loss 0.877\n",
      "Step 1100 | Loss 0.885\n",
      "\n",
      " the sand was alk the e roought dr, dre gooorked andr\n",
      "bed androot and at ck out of the candrowne ithe\n",
      "n beame re toldoof the plisidlengs tooo of that pprtliverness owith hone he he mor. \"He have cous \n",
      "\n",
      "Step 1110 | Loss 0.878\n",
      "Step 1120 | Loss 0.868\n",
      "Step 1130 | Loss 0.861\n",
      "Step 1140 | Loss 0.850\n",
      "Step 1150 | Loss 0.861\n",
      "Step 1160 | Loss 0.843\n",
      "Step 1170 | Loss 0.848\n",
      "Step 1180 | Loss 0.857\n",
      "Step 1190 | Loss 0.822\n",
      "Step 1200 | Loss 0.832\n",
      "\n",
      " the saming the port sidut trios\n",
      "ander, riestom werealy as, and as\n",
      "fe or of the lowechoum has inget toder.\n",
      "\n",
      "\n",
      "\"Wel, said Stepan Arkadyevitch frowned at Grinevitch's words, giving him tos itocin him in \n",
      "\n",
      "Step 1210 | Loss 0.830\n",
      "Step 1220 | Loss 0.847\n",
      "Step 1230 | Loss 0.854\n",
      "Step 1240 | Loss 0.844\n",
      "Step 1250 | Loss 0.823\n",
      "Step 1260 | Loss 0.826\n",
      "Step 1270 | Loss 0.807\n",
      "Step 1280 | Loss 0.818\n",
      "Step 1290 | Loss 0.787\n",
      "Step 1300 | Loss 0.807\n",
      "\n",
      " the sager doo of the boardrom of of the brarom blood the\n",
      "ooger-an sofes of the in prtlan whown st he leve men of the se set wo imetheme\n",
      "toos the iduting him prtionat tivery che onsat thoom. \"We that \n",
      "\n",
      "Step 1310 | Loss 0.815\n",
      "Step 1320 | Loss 0.777\n",
      "Step 1330 | Loss 0.776\n",
      "Step 1340 | Loss 0.788\n",
      "Step 1350 | Loss 0.800\n",
      "Step 1360 | Loss 0.768\n",
      "Step 1370 | Loss 0.798\n",
      "Step 1380 | Loss 0.779\n",
      "Step 1390 | Loss 0.762\n",
      "Step 1400 | Loss 0.738\n",
      "\n",
      " the dok a the dooo anok a ding ea yond you a thea the\n",
      "sexpid trion lang mereasestes. \"Whe\n",
      "\n",
      "\"Well, ll, hoow you f ce ce that d thereeprferfentment ew ha ered of his ouneart. Till tthe of the\n",
      "mersenthe\n",
      "\n",
      "Step 1410 | Loss 0.758\n",
      "Step 1420 | Loss 0.757\n",
      "Step 1430 | Loss 0.748\n",
      "Step 1440 | Loss 0.743\n",
      "Step 1450 | Loss 0.736\n",
      "Step 1460 | Loss 0.730\n",
      "Step 1470 | Loss 0.738\n",
      "Step 1480 | Loss 0.704\n",
      "Step 1490 | Loss 0.731\n",
      "Step 1500 | Loss 0.715\n",
      "\n",
      " the district to erall ar teve ming the show that here se to.\"\n",
      "\n",
      "\"No,\" answered,\" Levin, ad \" aid state s though to he had sadoread gner him ding to\n",
      "\n",
      "unoto the dooork abe in.\n",
      "\n",
      "\"Whaven sat do deperoin l\n",
      "\n",
      "Step 1510 | Loss 0.700\n",
      "Step 1520 | Loss 0.717\n",
      "Step 1530 | Loss 0.706\n",
      "Step 1540 | Loss 0.708\n",
      "Step 1550 | Loss 0.709\n",
      "Step 1560 | Loss 0.715\n",
      "Step 1570 | Loss 0.699\n",
      "Step 1580 | Loss 0.711\n",
      "Step 1590 | Loss 0.716\n",
      "Step 1600 | Loss 0.707\n",
      "\n",
      " a the family tham as the toncat Levin be the bans the the dayof he ham ad frend the portatencers of ee exof hirty youseser Evind thest\n",
      "sher ong pand the sof hirthat e waselif tere bons w, and the lif\n",
      "\n",
      "Step 1610 | Loss 0.715\n",
      "Step 1620 | Loss 0.697\n",
      "Step 1630 | Loss 0.696\n",
      "Step 1640 | Loss 0.683\n",
      "Step 1650 | Loss 0.684\n",
      "Step 1660 | Loss 0.674\n",
      "Step 1670 | Loss 0.664\n",
      "Step 1680 | Loss 0.682\n",
      "Step 1690 | Loss 0.691\n",
      "Step 1700 | Loss 0.683\n",
      "\n",
      " a the fousehim unearng ther sof the was day, had as thoughir wer abslovert on the se fort staking a chumsh touder. \"I he\n",
      "Shtchtcherery Levin her comed bere an coved with dis beroth the Konsky dis mev\n",
      "\n",
      "Step 1710 | Loss 0.667\n",
      "Step 1720 | Loss 0.677\n",
      "Step 1730 | Loss 0.675\n",
      "Step 1740 | Loss 0.665\n",
      "Step 1750 | Loss 0.659\n",
      "Step 1760 | Loss 0.645\n",
      "Step 1770 | Loss 0.679\n",
      "Step 1780 | Loss 0.650\n",
      "Step 1790 | Loss 0.686\n",
      "Step 1800 | Loss 0.670\n",
      "\n",
      " the sto at cevery sing Prethary could had hat p\n",
      "be off wen of thave\n",
      "therge lfened, caugeing din han ce din to been way tons to sel pecalerived ble, with al this he ad imil ve not use of hiseld, whad\n",
      "\n",
      "\n",
      "Step 1810 | Loss 0.664\n",
      "Step 1820 | Loss 0.664\n",
      "Step 1830 | Loss 0.654\n",
      "Step 1840 | Loss 0.642\n",
      "Step 1850 | Loss 0.651\n",
      "Step 1860 | Loss 0.664\n",
      "Step 1870 | Loss 0.665\n",
      "Step 1880 | Loss 0.662\n",
      "Step 1890 | Loss 0.662\n",
      "Step 1900 | Loss 0.654\n",
      "\n",
      " the ste onthe maly he of hir him, anding sedeed octocins wonthas e in\n",
      "the maring of thant pheross ant of hime the has re sund of that heyener ad had e aself could regard him as worthy\n",
      "of her.\n",
      "\n",
      "After \n",
      "\n",
      "Step 1910 | Loss 0.669\n",
      "Step 1920 | Loss 0.653\n",
      "Step 1930 | Loss 0.642\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1940 | Loss 0.643\n",
      "Step 1950 | Loss 0.635\n",
      "Step 1960 | Loss 0.629\n",
      "Step 1970 | Loss 0.620\n",
      "Step 1980 | Loss 0.619\n",
      "Step 1990 | Loss 0.629\n",
      "Step 2000 | Loss 0.645\n",
      "\n",
      " a so shorthat crrusie fare ary our ld nother she of his ring sof phr'senssonserssencions of his nsartars apherss was it\n",
      "imaplilerten was ther he thad of uld been become and moto anse in ato make an o\n",
      "\n",
      "Step 2010 | Loss 0.648\n",
      "Step 2020 | Loss 0.624\n",
      "Step 2030 | Loss 0.617\n",
      "Step 2040 | Loss 0.618\n",
      "Step 2050 | Loss 0.606\n",
      "Step 2060 | Loss 0.587\n",
      "Step 2070 | Loss 0.598\n",
      "Step 2080 | Loss 0.603\n",
      "Step 2090 | Loss 0.596\n",
      "Step 2100 | Loss 0.588\n",
      "\n",
      " that he country, and so istenter Englivesh broth ther prother they woung would she dis no theder o they und not be\n",
      "brother dincess; hand n life whad the m ato ell\n",
      "sensament of sort or be ar ther\n",
      "los \n",
      "\n",
      "Step 2110 | Loss 0.601\n",
      "Step 2120 | Loss 0.601\n",
      "Step 2130 | Loss 0.590\n",
      "Step 2140 | Loss 0.576\n",
      "Step 2150 | Loss 0.601\n",
      "Step 2160 | Loss 0.612\n",
      "Step 2170 | Loss 0.601\n",
      "Step 2180 | Loss 0.599\n",
      "Step 2190 | Loss 0.603\n",
      "Step 2200 | Loss 0.592\n",
      "\n",
      " the sundest at it on to deat co.\n",
      "\n",
      "He cambult are was funding on the profirt Konstanttance wo as blove a sa d himself, milerusal a d ance at of exend isterence his of sled har agreas him a n an the se\n",
      "\n",
      "Step 2210 | Loss 0.599\n",
      "Step 2220 | Loss 0.568\n",
      "Step 2230 | Loss 0.579\n",
      "Step 2240 | Loss 0.585\n",
      "Step 2250 | Loss 0.587\n",
      "Step 2260 | Loss 0.595\n",
      "Step 2270 | Loss 0.583\n",
      "Step 2280 | Loss 0.588\n",
      "Step 2290 | Loss 0.577\n",
      "Step 2300 | Loss 0.590\n",
      "\n",
      " and his cous hads would ncome with\n",
      "h him about of milighy and s beeemond therer\n",
      "shand ofes hisker dest her, skated to co snsiout se him a it andea of ining looy soat s surined hir\n",
      "sharencention the h\n",
      "\n",
      "Step 2310 | Loss 0.592\n",
      "Step 2320 | Loss 0.595\n",
      "Step 2330 | Loss 0.587\n",
      "Step 2340 | Loss 0.573\n",
      "Step 2350 | Loss 0.576\n",
      "Step 2360 | Loss 0.579\n",
      "Step 2370 | Loss 0.562\n",
      "Step 2380 | Loss 0.584\n",
      "Step 2390 | Loss 0.588\n",
      "Step 2400 | Loss 0.588\n",
      "\n",
      " the im, read reabot\n",
      "loved frskates, pting to and the sun, swith lir the fat\n",
      "sence etunce unceseized of the ressiong sledges as ther, skated by her, even spoke\n",
      "to her, and were happy, quite apart from\n",
      "\n",
      "Step 2410 | Loss 0.561\n",
      "Step 2420 | Loss 0.571\n",
      "Step 2430 | Loss 0.559\n",
      "Step 2440 | Loss 0.578\n",
      "Step 2450 | Loss 0.571\n",
      "Step 2460 | Loss 0.582\n",
      "Step 2470 | Loss 0.577\n",
      "Step 2480 | Loss 0.562\n",
      "Step 2490 | Loss 0.564\n",
      "Step 2500 | Loss 0.561\n",
      "\n",
      " he sun, awards by her, we's to seeem?\" Levin had he now!\" dustered going let I dono the der. Aff---bee to persseel witht wed oug oved, with was was were as sasy lippleace, at ongized, s and\n",
      "buther, w\n",
      "\n",
      "Step 2510 | Loss 0.547\n",
      "Step 2520 | Loss 0.556\n",
      "Step 2530 | Loss 0.548\n",
      "Step 2540 | Loss 0.552\n",
      "Step 2550 | Loss 0.563\n",
      "Step 2560 | Loss 0.541\n",
      "Step 2570 | Loss 0.545\n",
      "Step 2580 | Loss 0.559\n",
      "Step 2590 | Loss 0.553\n",
      "Step 2600 | Loss 0.545\n",
      "\n",
      " the skating-ground, and kept saying to\n",
      "himself--\"You mustn't be excited, you must be calm. What's the matter\n",
      "with you? What do you want? Be quiet, stupid,\" he conjured his heart.\n",
      "And the more he trie\n",
      "\n",
      "Step 2610 | Loss 0.549\n",
      "Step 2620 | Loss 0.556\n",
      "Step 2630 | Loss 0.558\n",
      "Step 2640 | Loss 0.544\n",
      "Step 2650 | Loss 0.551\n",
      "Step 2660 | Loss 0.555\n",
      "Step 2670 | Loss 0.536\n",
      "Step 2680 | Loss 0.539\n",
      "Step 2690 | Loss 0.523\n",
      "Step 2700 | Loss 0.530\n",
      "\n",
      " her, smile, which always transported Levin to an enchanted\n",
      "world, where he felt himself softened and tender, as he remembered\n",
      "himself in some days of his early childhood.\n",
      "\n",
      "\"Have you been here long?\" \n",
      "\n",
      "Step 2710 | Loss 0.529\n",
      "Step 2720 | Loss 0.548\n",
      "Step 2730 | Loss 0.549\n",
      "Step 2740 | Loss 0.543\n",
      "Step 2750 | Loss 0.538\n",
      "Step 2760 | Loss 0.538\n",
      "Step 2770 | Loss 0.538\n",
      "Step 2780 | Loss 0.524\n",
      "Step 2790 | Loss 0.540\n",
      "Step 2800 | Loss 0.540\n",
      "\n",
      " she cone motheld the fellen himself\n",
      "a soff hard he evid prectaione skated off,\n",
      "laughing.\n",
      "\n",
      "\"How splendid, how nice he is!\" Kitty was thinking at that time, as she\n",
      "came out of the pavilion with Mlle. L\n",
      "\n",
      "Step 2810 | Loss 0.537\n",
      "Step 2820 | Loss 0.549\n",
      "Step 2830 | Loss 0.537\n",
      "Step 2840 | Loss 0.525\n",
      "Step 2850 | Loss 0.508\n",
      "Step 2860 | Loss 0.511\n",
      "Step 2870 | Loss 0.508\n",
      "Step 2880 | Loss 0.521\n",
      "Step 2890 | Loss 0.504\n",
      "Step 2900 | Loss 0.513\n",
      "\n",
      " the conver to mys, and he be laughing the with he of trsh with\n",
      "he st same skated without\n",
      "effort, as it were, by simple exercise of will, increasing and\n",
      "slackening speed and turning his course. He app\n",
      "\n",
      "Step 2910 | Loss 0.505\n",
      "Step 2920 | Loss 0.516\n",
      "Step 2930 | Loss 0.524\n",
      "Step 2940 | Loss 0.508\n",
      "Step 2950 | Loss 0.511\n",
      "Step 2960 | Loss 0.511\n",
      "Step 2970 | Loss 0.521\n",
      "Step 2980 | Loss 0.504\n",
      "Step 2990 | Loss 0.513\n",
      "Step 3000 | Loss 0.509\n",
      "\n",
      " sheard stabled ing drined about him in evening coats, bearing napkins. Bowing to right\n",
      "and left to the people he met, and here as everywher was a you her\n",
      "solves to too ther of stumble, twice struck o\n",
      "\n",
      "Step 3010 | Loss 0.501\n",
      "Step 3020 | Loss 0.513\n",
      "Step 3030 | Loss 0.496\n",
      "Step 3040 | Loss 0.503\n",
      "Step 3050 | Loss 0.488\n",
      "Step 3060 | Loss 0.485\n",
      "Step 3070 | Loss 0.473\n",
      "Step 3080 | Loss 0.481\n",
      "Step 3090 | Loss 0.479\n",
      "Step 3100 | Loss 0.485\n",
      "\n",
      " the conver the so quer titer\n",
      "hat for heavess.\n",
      "\"Wit, it befor hand yout gor But hist he Chames.\n",
      "\n",
      "\"What stalll that the so be ar,\" he cand uppressell if cit of at voice saying, \"Good-bye till this even\n",
      "\n",
      "Step 3110 | Loss 0.463\n",
      "Step 3120 | Loss 0.474\n",
      "Step 3130 | Loss 0.472\n",
      "Step 3140 | Loss 0.489\n",
      "Step 3150 | Loss 0.458\n",
      "Step 3160 | Loss 0.456\n",
      "Step 3170 | Loss 0.466\n",
      "Step 3180 | Loss 0.465\n",
      "Step 3190 | Loss 0.470\n",
      "Step 3200 | Loss 0.469\n",
      "\n",
      " they fresh?\"\n",
      "\n",
      "\"Only arrived yesterday.\"\n",
      "\n",
      "\"Well, then, how if we were to begin with oysters, and so change the\n",
      "whole program? Eh?\"\n",
      "\n",
      "\"It's all the same to me. I should like cabbage soup and porridge be\n",
      "\n",
      "Step 3210 | Loss 0.475\n",
      "Step 3220 | Loss 0.467\n",
      "Step 3230 | Loss 0.474\n",
      "Step 3240 | Loss 0.463\n",
      "Step 3250 | Loss 0.453\n",
      "Step 3260 | Loss 0.476\n",
      "Step 3270 | Loss 0.482\n",
      "Step 3280 | Loss 0.461\n",
      "Step 3290 | Loss 0.453\n",
      "Step 3300 | Loss 0.457\n",
      "\n",
      " in glass of Chablis, nectings, and thand the grmesesin his the napskinew hile.\n",
      "\n",
      "\"I?\" said Stepan Arkadyevitch, \"there's nothing I desire so much as\n",
      "that--nothing! It would be the best thing that coul\n",
      "\n",
      "Step 3310 | Loss 0.453\n",
      "Step 3320 | Loss 0.454\n",
      "Step 3330 | Loss 0.452\n",
      "Step 3340 | Loss 0.424\n",
      "Step 3350 | Loss 0.449\n",
      "Step 3360 | Loss 0.444\n",
      "Step 3370 | Loss 0.494\n",
      "Step 3380 | Loss 0.473\n",
      "Step 3390 | Loss 0.456\n",
      "Step 3400 | Loss 0.452\n",
      "\n",
      " one comfort oome.\"\n",
      "\n",
      "\"It's not mul my musto stall the me oble that a bother abothers it toksen on of me. I went away, you see, because I made up my mind\n",
      "that it could never be, you understand, as a ha\n",
      "\n",
      "Step 3410 | Loss 0.457\n",
      "Step 3420 | Loss 0.461\n",
      "Step 3430 | Loss 0.454\n",
      "Step 3440 | Loss 0.460\n",
      "Step 3450 | Loss 0.440\n",
      "Step 3460 | Loss 0.433\n",
      "Step 3470 | Loss 0.445\n",
      "Step 3480 | Loss 0.458\n",
      "Step 3490 | Loss 0.457\n",
      "Step 3500 | Loss 0.439\n",
      "\n",
      " me sometimes. That will be awful for me, and for her\n",
      "too.\"\n",
      "\n",
      "\"Oh, well, anyway there's nothing awful in it for a girl. Every girl's\n",
      "proud of an offer.\"\n",
      "\n",
      "\"Yes, every girl, but not she.\"\n",
      "\n",
      "Stepan Arkadye\n",
      "\n",
      "Step 3510 | Loss 0.445\n",
      "Step 3520 | Loss 0.463\n",
      "Step 3530 | Loss 0.438\n",
      "Step 3540 | Loss 0.446\n",
      "Step 3550 | Loss 0.444\n",
      "Step 3560 | Loss 0.458\n",
      "Step 3570 | Loss 0.456\n",
      "Step 3580 | Loss 0.460\n",
      "Step 3590 | Loss 0.460\n",
      "Step 3600 | Loss 0.437\n",
      "\n",
      " one conversation coulds ther subttle theits, ittles all and the ried that thate ar mothe sort of eme an only the other. And those who only know\n",
      "the non-platonic love have no need to the not the cres \n",
      "\n",
      "Step 3610 | Loss 0.452\n",
      "Step 3620 | Loss 0.446\n",
      "Step 3630 | Loss 0.444\n",
      "Step 3640 | Loss 0.446\n",
      "Step 3650 | Loss 0.445\n",
      "Step 3660 | Loss 0.439\n",
      "Step 3670 | Loss 0.452\n",
      "Step 3680 | Loss 0.451\n",
      "Step 3690 | Loss 0.459\n",
      "Step 3700 | Loss 0.468\n",
      "\n",
      " one conversation hing him about acquante mand her mor toold the con The wing one hand sat con winter, as from\n",
      "bed had intar\n",
      "and that that pater bicature, love, which was that praing to wanted\n",
      "done hi\n",
      "\n",
      "Step 3710 | Loss 0.465\n",
      "Step 3720 | Loss 0.466\n",
      "Step 3730 | Loss 0.453\n",
      "Step 3740 | Loss 0.454\n",
      "Step 3750 | Loss 0.439\n",
      "Step 3760 | Loss 0.461\n",
      "Step 3770 | Loss 0.460\n",
      "Step 3780 | Loss 0.460\n",
      "Step 3790 | Loss 0.449\n",
      "Step 3800 | Loss 0.467\n",
      "\n",
      " of love, which you remember Plato\n",
      "defines in his Banquet, served as the test of men. Some men only\n",
      "understand one sort, and some only the other. And those who only know\n",
      "the non-platonic love have no \n",
      "\n",
      "Step 3810 | Loss 0.461\n",
      "Step 3820 | Loss 0.453\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3830 | Loss 0.437\n",
      "Step 3840 | Loss 0.443\n",
      "Step 3850 | Loss 0.450\n",
      "Step 3860 | Loss 0.464\n",
      "Step 3870 | Loss 0.454\n",
      "Step 3880 | Loss 0.451\n",
      "Step 3890 | Loss 0.444\n",
      "Step 3900 | Loss 0.452\n",
      "\n",
      " the young men who\n",
      "danced at the Moscow balls being almost all in love with Kitty, two\n",
      "serious suitors had already this first winter made their appearance:\n",
      "Levin, and immediately after his departure, \n",
      "\n",
      "Step 3910 | Loss 0.444\n",
      "Step 3920 | Loss 0.435\n",
      "Step 3930 | Loss 0.440\n",
      "Step 3940 | Loss 0.427\n",
      "Step 3950 | Loss 0.451\n",
      "Step 3960 | Loss 0.449\n",
      "Step 3970 | Loss 0.459\n",
      "Step 3980 | Loss 0.458\n",
      "Step 3990 | Loss 0.448\n",
      "Step 4000 | Loss 0.428\n",
      "\n",
      " as exireestand, but the mor Plat Levin was not them.\n",
      "\n",
      "\"Insad ted thinking of his own affairs, and they had bottly cat of that he armother. Oblonsky had more than once experienced\n",
      "this extreme sense o\n",
      "\n",
      "Step 4010 | Loss 0.446\n",
      "Step 4020 | Loss 0.447\n",
      "Step 4030 | Loss 0.438\n",
      "Step 4040 | Loss 0.435\n",
      "Step 4050 | Loss 0.435\n",
      "Step 4060 | Loss 0.432\n",
      "Step 4070 | Loss 0.440\n",
      "Step 4080 | Loss 0.430\n",
      "Step 4090 | Loss 0.414\n",
      "Step 4100 | Loss 0.421\n",
      "\n",
      " the army\n",
      "and at court, and a fascinating man. Nothing better could be wished for.\n",
      "\n",
      "Vronsky openly flirted with Kitty at balls, danced with her, and came\n",
      "continually to the house, consequently there c\n",
      "\n",
      "Step 4110 | Loss 0.418\n",
      "Step 4120 | Loss 0.401\n"
     ]
    }
   ],
   "source": [
    "sess = tf.Session()\n",
    "sess.run(tf.global_variables_initializer())\n",
    "while True:\n",
    "    try:\n",
    "        _, step, loss = sess.run([ops['train'], ops['global_step'], ops['loss']])\n",
    "    except tf.errors.OutOfRangeError:\n",
    "        break\n",
    "    else:\n",
    "        if step % params['display_step'] == 0 or step == 1:\n",
    "            print(\"Step %d | Loss %.3f\" % (step, loss))\n",
    "        if step % params['generate_step'] == 0 and step > 1:\n",
    "            ints = sess.run(ops['generate'])\n",
    "            print('\\n'+''.join([params['idx2char'][i] for i in ints])+'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
